probe AWS datacenter

S3: When initremote is given the name of a bucket that already exists,
automatically set datacenter to the right value, rather than needing it to
be explicitly set.

This needs aws-0.23. But, initremote stores the datacenter value, so
a remote set up this way can be used with git-annex built with an older aws.

This is not done when signature=anonymous, because in that case,
using AWS.defaultRegion works fine for accessing buckets on other
datacenters.

It feels a bit round-about to need to do this probing. But without it,
the problem seems to be that, with a v4 signature, the location constraint
is included in the Authorization header. When that is the wrong location,
AWS S3 rejects it. I do wonder though if there is an easier way that I
am currently missing.

Sponsored-by: Dartmouth College's DANDI project

diff --git a/CHANGELOG b/CHANGELOG
index 434d2c160c93507e1b5682470feddfbd09fba75a..6fb6f99d6b4b8c5ea5bda86b9cc4757e1701b58d 100644 (file)
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -10,6 +10,9 @@ git-annex (10.20250722) UNRELEASED; urgency=medium
     a different S3 host, the default remains signature=v2.
   * webapp: Support setting up S3 buckets in regions that need v4
     signatures.
+  * S3: When initremote is given the name of a bucket that already exists,
+    automatically set datacenter to the right value, rather than needing it
+    to be explicitly set. (This needs aws-0.23)
 
  -- Joey Hess <id@joeyh.name>  Wed, 30 Jul 2025 13:45:42 -0400
 

diff --git a/Remote/S3.hs b/Remote/S3.hs
index 1fa8d44cd3e9eff60c588f25343d613dec452b98..4da5ff3f36caead5d2afa78ce6083b67068bd8d9 100644 (file)
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -193,7 +193,7 @@ gen r u rc gc rs = do
        c <- parsedRemoteConfig remote rc
        cst <- remoteCost gc c expensiveRemoteCost
        info <- extractS3Info c
-       hdl <- mkS3HandleVar c gc u
+       hdl <- mkS3HandleVar False c gc u
        magic <- liftIO initMagicMime
        return $ new c cst info hdl magic
   where
@@ -287,7 +287,17 @@ s3Setup'  ss u mcreds c gc
                        =<< configParser remote c'
                c'' <- if isAnonymous pc
                        then pure c'
-                       else setRemoteCredPair ss encsetup pc gc (AWS.creds u) mcreds
+                       else do
+                               v <- setRemoteCredPair ss encsetup pc gc (AWS.creds u) mcreds
+                               if M.member datacenterField c || M.member regionField c
+                                       then return v
+                                       -- Check if a bucket with this name
+                                       -- already exists, and if so, use
+                                       -- that location, rather than the
+                                       -- default datacenterField.
+                                       else getBucketLocation pc gc u >>= return . \case
+                                               Nothing -> v
+                                               Just loc -> M.insert datacenterField (Proposed $ T.unpack loc) v
                pc' <- either giveup return . parseRemoteConfig c''
                        =<< configParser remote c''
                info <- extractS3Info pc'
@@ -322,7 +332,7 @@ s3Setup'  ss u mcreds c gc
                        =<< configParser remote archiveconfig
                info <- extractS3Info pc'
                checkexportimportsafe pc' info
-               hdl <- mkS3HandleVar pc' gc u
+               hdl <- mkS3HandleVar False pc' gc u
                withS3HandleOrFail u hdl $
                        writeUUIDFile pc' u info
                use archiveconfig pc' info
@@ -775,6 +785,22 @@ checkPresentExportWithContentIdentifierS3 hv r info _k loc knowncids =
   where
        o = T.pack $ bucketExportLocation info loc
 
+getBucketLocation :: ParsedRemoteConfig -> RemoteGitConfig -> UUID -> Annex (Maybe S3.LocationConstraint)
+getBucketLocation c gc u = do
+#if MIN_VERSION_aws(0,23,0)
+       info <- extractS3Info c
+       let info' = info { region = Nothing, host = Nothing }
+       -- Force anonymous access, because this API call does not work
+       -- when used in an authenticated context.
+       hdl <- mkS3HandleVar True c gc u
+       withS3HandleOrFail u hdl $ \h -> do
+               r <- liftIO $ tryNonAsync $ runResourceT $
+                       sendS3Handle h (S3.getBucketLocation $ bucket info')
+               return $ either (const Nothing) (Just . S3.gblrLocationConstraint) r
+#else
+       return Nothing
+#endif
+
 {- Generate the bucket if it does not already exist, including creating the
  - UUID file within the bucket.
  -
@@ -786,7 +812,7 @@ genBucket :: ParsedRemoteConfig -> RemoteGitConfig -> UUID -> Annex ()
 genBucket c gc u = do
        showAction "checking bucket"
        info <- extractS3Info c
-       hdl <- mkS3HandleVar c gc u
+       hdl <- mkS3HandleVar False c gc u
        withS3HandleOrFail u hdl $ \h ->
                go info h =<< checkUUIDFile c u info h
   where
@@ -896,9 +922,9 @@ giveupS3HandleProblem S3HandleAnonymousOldAws _ =
 
 {- Prepares a S3Handle for later use. Does not connect to S3 or do anything
  - else expensive. -}
-mkS3HandleVar :: ParsedRemoteConfig -> RemoteGitConfig -> UUID -> Annex S3HandleVar
-mkS3HandleVar c gc u = liftIO $ newTVarIO $ Left $
-       if isAnonymous c
+mkS3HandleVar :: Bool -> ParsedRemoteConfig -> RemoteGitConfig -> UUID -> Annex S3HandleVar
+mkS3HandleVar forceanonymous c gc u = liftIO $ newTVarIO $ Left $
+       if forceanonymous || isAnonymous c
                then 
 #if MIN_VERSION_aws(0,23,0)
                        go =<< liftIO AWS.anonymousCredentials
@@ -1380,7 +1406,7 @@ enableBucketVersioning ss info c gc u = do
   where
        enableversioning b = do
                showAction "checking bucket versioning"
-               hdl <- mkS3HandleVar c gc u
+               hdl <- mkS3HandleVar False c gc u
                let setversioning = S3.putBucketVersioning b S3.VersioningEnabled
                withS3HandleOrFail u hdl $ \h ->
 #if MIN_VERSION_aws(0,24,3)

diff --git a/doc/bugs/fails_to_authenticate_into_S3_for_initremote__63__/comment_5_4f41ad125c1af26cb94bd40c61cfcd9f._comment b/doc/bugs/fails_to_authenticate_into_S3_for_initremote__63__/comment_5_4f41ad125c1af26cb94bd40c61cfcd9f._comment
new file mode 100644 (file)
index 0000000..fc1f37b
--- /dev/null
+++ b/doc/bugs/fails_to_authenticate_into_S3_for_initremote__63__/comment_5_4f41ad125c1af26cb94bd40c61cfcd9f._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2025-08-13T18:22:53Z"
+ content="""
+I got GetBucketLocation to work, although only when git-annex is built with
+aws-0.23 or newer.
+
+       joey@darkstar:~/tmp/a9>git-annex initremote s3-originnew  type=S3 importtree=yes encryption=none autoenable=true bucket=dandiarchive fileprefix=zarr-checksums/2ac71edb-738c-40ac-bd8c-8ca985adaa12/
+       initremote s3-originnew (checking bucket...) ok
+       (recording state in git...)
+"""]]

diff --git a/doc/special_remotes/S3.mdwn b/doc/special_remotes/S3.mdwn
index 96dfdb56d72bd501bc4e0938df496c5b4b648074..935839a815e5d13cca2d546bc855cc5afd9e4641 100644 (file)
--- a/doc/special_remotes/S3.mdwn
+++ b/doc/special_remotes/S3.mdwn
@@ -35,11 +35,10 @@ the S3 remote.
   embedcreds without gpg encryption.
 
 * `datacenter` - Specifies which Amazon datacenter
-  to use for the bucket. Defaults to "US". Other values include "EU"
-  (which is EU/Ireland), "us-west-1", "us-west-2", "ap-southeast-1",
-  "ap-southeast-2", and "sa-east-1". See Amazon's documentation for a
-  complete list. Configuring this is equivilant to configuring both
-  `host` and `region`.
+  to use when creating a bucket. Defaults to "US". Other values include "EU"
+  (which is EU/Ireland), "us-west-1", "us-west-2", etc. See Amazon's
+  documentation for a complete list. Configuring this is equivilant to
+  configuring both `host` and `region`.
 
 * `storageclass` - Default is "STANDARD".  
   Consult S3 provider documentation for pricing details and available